core: Support building with OpenSSL for checksums

Add an OpenSSL backend to the checksum input stream, which is where we do a lot
of checksumming (object commit, static deltas).

The raw OpenSSL performance is
[approximately double](https://gist.github.com/cgwalters/169349fd1c06fd4fb4d3a7ce33303222) on
my laptop; not only does OpenSSL have e.g. hand-tuned x86_64 assembly, the
current implementation uses the
[Intel SHA extensions](https://en.wikipedia.org/wiki/Intel_SHA_extensions).

Another reason to do this is I was idly thinking about adding
[Curve25519](https://en.wikipedia.org/wiki/Curve25519) signatures (like e.g.
Alpine does) instead of/in addition to GPG.  The rationale for that is
that GPG is pretty heavyweight, both in code footprint and the simple
fact that EC keys are way smaller.

I didn't benchmark ostree with this; we have bigger performance problems
really like the fact we just malloc way too much.  But, it's a step
in the right direction I think in combination with the libcurl work
where we're linking to openssl anyways.

Closes: #738
Approved by: jlebon

diff --git a/.redhat-ci.yml b/.redhat-ci.yml
index 74778f8acea1c734884ad6802f7e57782746bc76..8073ed9c4ead577e4916ab6fe63f2f3f6ab994d9 100644 (file)
--- a/.redhat-ci.yml
+++ b/.redhat-ci.yml
@@ -76,10 +76,11 @@ tests:
 inherit: true
 required: true
 
-context: curl
+context: curl-openssl
 
 packages:
   - pkgconfig(libcurl)
+  - pkgconfig(openssl)
 
 build:
     config-opts: >
@@ -88,6 +89,7 @@ build:
       --enable-installed-tests
       --enable-gtk-doc
       --with-curl
+      --with-openssl
 
 tests:
   - make check

diff --git a/configure.ac b/configure.ac
index be1e0e13146c7afaabe0e2445482e978863f09d6..0afd87e10de04bdc8ff064f4c54fbbfd28b33058 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -298,6 +298,23 @@ AS_IF([ test x$with_smack = xyes], [
 ])
 AM_CONDITIONAL(USE_SMACK, test $with_smack != no)
 
+dnl begin openssl
+OPENSSL_DEPENDENCY="libselinux >= 1.0.1"
+AC_ARG_WITH(openssl,
+AS_HELP_STRING([--with-openssl], [Enable use of OpenSSL (checksums)]),
+:, with_openssl=no)
+
+AS_IF([ test x$with_openssl != xno ], [
+      PKG_CHECK_MODULES(OT_DEP_OPENSSL, $OPENSSL_DEPENDENCY)
+      AC_DEFINE([HAVE_OPENSSL], 1, [Define if we have openssl])
+      with_openssl=yes
+], [
+      with_openssl=no
+])
+if test x$with_openssl != xno; then OSTREE_FEATURES="$OSTREE_FEATURES openssl"; fi
+AM_CONDITIONAL(USE_OPENSSL, test $with_openssl != no)
+dnl end openssl
+
 dnl This is what is in RHEL7.2 right now, picking it arbitrarily
 LIBMOUNT_DEPENDENCY="mount >= 2.23.0"
 
@@ -430,6 +447,7 @@ echo "
     HTTP backend:                                 $fetcher_backend
     \"ostree trivial-httpd\":                       $enable_trivial_httpd_cmdline
     SELinux:                                      $with_selinux
+    OpenSSL (checksums):                          $with_openssl
     systemd:                                      $have_libsystemd
     libmount:                                     $with_libmount
     libarchive (parse tar files directly):        $with_libarchive

diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index db03c034d1510916d9711a53818ab700a009bfec..4d7b431b1d1ecd951a8296039b1041272334f5a1 100644 (file)
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -1351,16 +1351,7 @@ void
 ostree_checksum_inplace_from_bytes (const guchar *csum,
                                     char         *buf)
 {
-  static const gchar hexchars[] = "0123456789abcdef";
-  guint i, j;
-
-  for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i++, j += 2)
-    {
-      guchar byte = csum[i];
-      buf[j] = hexchars[byte >> 4];
-      buf[j+1] = hexchars[byte & 0xF];
-    }
-  buf[j] = '\0';
+  ot_bin2hex (buf, csum, OSTREE_SHA256_DIGEST_LEN);
 }
 
 /**

diff --git a/src/libotutil/ot-checksum-instream.c b/src/libotutil/ot-checksum-instream.c
index 14d5b5f0e6e37b8a4a44fc938ccf5027fb97d974..686554df321ab810dc58fd7173048b838f6ee762 100644 (file)
--- a/src/libotutil/ot-checksum-instream.c
+++ b/src/libotutil/ot-checksum-instream.c
@@ -21,11 +21,21 @@
 #include "config.h"
 
 #include "ot-checksum-instream.h"
+#include "ot-checksum-utils.h"
+
+#ifdef HAVE_OPENSSL
+#include <openssl/evp.h>
+#endif
 
 G_DEFINE_TYPE (OtChecksumInstream, ot_checksum_instream, G_TYPE_FILTER_INPUT_STREAM)
 
 struct _OtChecksumInstreamPrivate {
+#ifdef HAVE_OPENSSL
+  EVP_MD_CTX *checksum;
+#else
+  GChecksumType checksum_type;
   GChecksum *checksum;
+#endif
 };
 
 static gssize   ot_checksum_instream_read         (GInputStream         *stream,
@@ -39,7 +49,11 @@ ot_checksum_instream_finalize (GObject *object)
 {
   OtChecksumInstream *self = (OtChecksumInstream*)object;
 
+#ifdef HAVE_OPENSSL
+  EVP_MD_CTX_destroy (self->priv->checksum);
+#else
   g_checksum_free (self->priv->checksum);
+#endif
 
   G_OBJECT_CLASS (ot_checksum_instream_parent_class)->finalize (object);
 }
@@ -60,8 +74,22 @@ static void
 ot_checksum_instream_init (OtChecksumInstream *self)
 {
   self->priv = G_TYPE_INSTANCE_GET_PRIVATE (self, OT_TYPE_CHECKSUM_INSTREAM, OtChecksumInstreamPrivate);
+}
 
+#ifdef HAVE_OPENSSL
+static const EVP_MD *
+gchecksum_type_to_openssl (GChecksumType checksum_type)
+{
+  switch (checksum_type)
+    {
+    case G_CHECKSUM_SHA256:
+      return EVP_sha256 ();
+    default:
+      /* If there's something else, fill in here */
+      g_assert_not_reached ();
+    }
 }
+#endif
 
 OtChecksumInstream *
 ot_checksum_instream_new (GInputStream    *base,
@@ -74,7 +102,18 @@ ot_checksum_instream_new (GInputStream    *base,
   stream = g_object_new (OT_TYPE_CHECKSUM_INSTREAM,
                          "base-stream", base,
                          NULL);
+
+  /* For now */
+  g_assert (checksum_type == G_CHECKSUM_SHA256);
+
+#ifdef HAVE_OPENSSL
+  stream->priv->checksum = EVP_MD_CTX_create ();
+  g_assert (stream->priv->checksum);
+  g_assert (EVP_DigestInit_ex (stream->priv->checksum, gchecksum_type_to_openssl (checksum_type), NULL));
+#else
   stream->priv->checksum = g_checksum_new (checksum_type);
+  stream->priv->checksum_type = checksum_type;
+#endif
 
   return (OtChecksumInstream*) (stream);
 }
@@ -96,7 +135,13 @@ ot_checksum_instream_read (GInputStream  *stream,
                              cancellable,
                              error);
   if (res > 0)
-    g_checksum_update (self->priv->checksum, buffer, res);
+    {
+#ifdef HAVE_OPENSSL
+      g_assert (EVP_DigestUpdate (self->priv->checksum, buffer, res));
+#else
+      g_checksum_update (self->priv->checksum, buffer, res);
+#endif
+    }
 
   return res;
 }
@@ -106,17 +151,29 @@ ot_checksum_instream_get_digest (OtChecksumInstream *stream,
                                  guint8          *buffer,
                                  gsize           *digest_len)
 {
+#ifdef HAVE_OPENSSL
+  unsigned len;
+  EVP_DigestFinal_ex (stream->priv->checksum, buffer, &len);
+  if (digest_len)
+    *digest_len = len;
+#else
   g_checksum_get_digest (stream->priv->checksum, buffer, digest_len);
+#endif
 }
 
 guint8*
 ot_checksum_instream_dup_digest (OtChecksumInstream *stream,
                                  gsize              *ret_len)
 {
-  gsize len = 32;
+#ifdef HAVE_OPENSSL
+  guint len;
+  guchar *ret = g_malloc0 (EVP_MAX_MD_SIZE);
+  g_assert (EVP_DigestFinal_ex (stream->priv->checksum, ret, &len));
+#else
+  gsize len = g_checksum_type_get_length (stream->priv->checksum_type);
   guchar *ret = g_malloc (len);
   g_checksum_get_digest (stream->priv->checksum, ret, &len);
-  g_assert (len == 32);
+#endif
   if (ret_len)
     *ret_len = len;
   return ret;
@@ -125,5 +182,14 @@ ot_checksum_instream_dup_digest (OtChecksumInstream *stream,
 char *
 ot_checksum_instream_get_string (OtChecksumInstream *stream)
 {
+#ifdef HAVE_OPENSSL
+  unsigned len;
+  guint8 csum[EVP_MAX_MD_SIZE];
+  g_assert (EVP_DigestFinal_ex (stream->priv->checksum, csum, &len));
+  char *buf = g_malloc (len * 2 + 1);
+  ot_bin2hex (buf, (guint8*)csum, len);
+  return buf;
+#else
   return g_strdup (g_checksum_get_string (stream->priv->checksum));
+#endif
 }

diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c
index 8d30bdc30770b8aa3bd56f3f2095e1108595efab..4b5c824a37e8e8c11018eb33d1c7af6f8fd24ab9 100644 (file)
--- a/src/libotutil/ot-checksum-utils.c
+++ b/src/libotutil/ot-checksum-utils.c
@@ -26,6 +26,22 @@
 
 #include <string.h>
 
+
+void
+ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len)
+{
+  static const gchar hexchars[] = "0123456789abcdef";
+  guint i, j;
+
+  for (i = 0, j = 0; i < len; i++, j += 2)
+    {
+      guchar byte = inbuf[i];
+      out_buf[j] = hexchars[byte >> 4];
+      out_buf[j+1] = hexchars[byte & 0xF];
+    }
+  out_buf[j] = '\0';
+}
+
 guchar *
 ot_csum_from_gchecksum (GChecksum  *checksum)
 {

diff --git a/src/libotutil/ot-checksum-utils.h b/src/libotutil/ot-checksum-utils.h
index 8b3a394e492867f235e2f71b9799a994a6ec3793..b580fc1c746d32b4c1fe916d8c36e1d3f2195037 100644 (file)
--- a/src/libotutil/ot-checksum-utils.h
+++ b/src/libotutil/ot-checksum-utils.h
@@ -26,6 +26,8 @@
 
 G_BEGIN_DECLS
 
+void ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len);
+
 guchar *ot_csum_from_gchecksum (GChecksum *checksum);
 
 gboolean ot_gio_write_update_checksum (GOutputStream  *out,